##########################################################################################
# Title: Build Gradient Dataset → Distance to BD vs Population & Built-up Density
# Section: 2 (Inputs → Gradients)
# Purpose: Consolidate per-city distance matrices (GHSL 100m grids) into a unified
#          dataset for population/built-up gradients within 20 km of business districts.
# IO:
#   IN : Processed_Data/Intermediate/Section_2/distance_matrix/<City>_100K_09_2020.rds
#   OUT: Processed_Data/Intermediate/Section_2/Gradients/gams_gradient.rds (and .csv)
##########################################################################################

rm(list = ls()); options(stringsAsFactors = FALSE)

suppressPackageStartupMessages({
  library(dplyr)
  library(readr)
  library(fs)
})

# ---- Project root (align with run_all.R) -------------------------------------
if (!requireNamespace("here", quietly = TRUE)) install.packages("here", quiet = TRUE)
ROOT     <- normalizePath(here::here(), winslash = "/")
IN_DIR   <- file.path(ROOT, "Processed_Data", "Intermediate", "Section_2", "distance_matrix")
MID_DIR  <- file.path(ROOT, "Processed_Data", "Intermediate", "Section_2", "Data_Gradients")
OUT_DIR  <- file.path(ROOT, "Processed_Data", "Intermediate", "Section_2", "Gradients")
dir.create(MID_DIR, recursive = TRUE, showWarnings = FALSE)
dir.create(OUT_DIR, recursive = TRUE, showWarnings = FALSE)

# ---- Cities ------------------------------------------------------------------
cities <- c(
  "Johannesburg","Capetown","Tshwane","Durban","NM_Bay","Nairobi","Lagos",
  "Abidjan","Cairo","London","Istanbul","Seoul","SaoPaulo","KualaLumpur",
  "Moscow","Mexico_City","Buenos_Aires","Shenzhen","Frankfurt","Zurich",
  "Bogota","Tehran","Dubai","New_York","Mumbai"
)

# If filenames deviate from the default pattern, map them here:
filename_map <- c(
  # "New_York" = "New_York_80K_04_2020.rds"
)

# ---- Helper: pick the first available column ---------------------------------
pick_col <- function(df, candidates) {
  hit <- candidates[candidates %in% names(df)]
  if (length(hit) == 0) return(rep(NA_real_, nrow(df)))
  out <- df[[hit[1]]]
  if (length(hit) > 1) for (j in hit[-1]) out <- dplyr::coalesce(out, df[[j]])
  out
}

# ---- Per-city processing -----------------------------------------------------
intermediate_paths <- list()

for (city in cities) {
  fname <- if (!is.null(filename_map[[city]])) filename_map[[city]] else paste0(city, "_100K_09_2020.rds")
  fpath <- file.path(IN_DIR, fname)
  if (!file.exists(fpath)) {
    warning(sprintf("Missing input for %s: %s", city, fpath))
    next
  }
  
  df <- readRDS(fpath)
  
  # Standardize columns
  POP   <- pick_col(df, c("GHS_POP_100_2020","pop_2020","population_2020","POP_2020","pop"))
  BUILT <- pick_col(df, c("GHS_BUILT_100m_2020","built_2020","built_100m_2020","GHS_BUILT_2020","built"))
  DIST  <- pick_col(df, c("min_dist_cbd","min_dist_bd","min_dist","distance_to_bd","dist_to_bd_m"))
  
  out <- tibble::tibble(
    city_name           = city,
    GHS_POP_100_2020    = as.numeric(POP),
    GHS_BUILT_100m_2020 = as.numeric(BUILT),
    min_dist_cbd        = as.numeric(DIST)
  ) %>%
    filter(
      !is.na(GHS_POP_100_2020),
      GHS_POP_100_2020 > 10,
      !is.na(min_dist_cbd),
      min_dist_cbd <= 20000
    ) %>%
    select(city_name, GHS_POP_100_2020, GHS_BUILT_100m_2020, min_dist_cbd)
  
  # Write per-city intermediates
  city_rds <- file.path(MID_DIR, paste0(city, ".rds"))
  saveRDS(out, city_rds)
  write_csv(out, fs::path_ext_set(city_rds, "csv"))
  intermediate_paths[[city]] <- city_rds
}

# ---- Combine intermediates ---------------------------------------------------
all_list <- lapply(intermediate_paths, function(p) if (file.exists(p)) readRDS(p) else NULL)
all_list <- all_list[!vapply(all_list, is.null, logical(1))]
if (length(all_list) == 0) stop("No intermediates were created. Check input filenames/patterns.")

gams_gradient <- dplyr::bind_rows(all_list) %>%
  mutate(
    south_africa = city_name %in% c("Capetown","Johannesburg","Durban","Tshwane","NM_Bay"),
    other_africa = city_name %in% c("Lagos","Nairobi","Abidjan","Cairo")
  )

# ---- Write final -------------------------------------------------------------
final_rds <- file.path(OUT_DIR, "gams_gradient.rds")
saveRDS(gams_gradient, final_rds)
write_csv(gams_gradient, fs::path_ext_set(final_rds, "csv"))

message("✅ Done. Final gradient dataset: ", normalizePath(final_rds))

